# -*- coding: utf-8 -*-
'''
Created on 14-12-2012

@author: LONG HOANG GIANG
'''

from CrawlerLib2 import commonlib, html2text
from Model.storymodelv3 import StoryModel
from urlparse import urljoin
import datetime
import os
import re


def getDetail(url, title):
    data = []
    c = 1
    titlefm = title
    while True:
        tree = commonlib.loadweb(url).build_tree()
        contentNode = tree.xpath("//div[@id='page']")[0]
        commonlib.Etree.clean(contentNode.xpath(".//form/span[@class='content_rating']/..")[0])
        tNode = contentNode.xpath(".//h4")
        if len(tNode) > 0:
            commonlib.Etree.clean(tNode[0])
        commonlib.Etree.clean_elements(contentNode, "p", {"class": "buttonheading"})
        commonlib.Etree.clean_elements(contentNode, "div", {"class": "pagenavcounter"})
        commonlib.Etree.clean_elements(contentNode, "table", {"class": "contenttoc"})
        
        nextNode = contentNode.xpath(".//div[@class='pagenavbar']//a[contains(., 'Tiếp theo')]".decode('utf-8'))
        br = len(nextNode) == 0
        if not br:
            nextUrl = nextNode[0].get('href', '')
            if nextUrl.strip() == '': br = True
            url = urljoin(url, nextUrl)
        
        commonlib.Etree.clean_elements(contentNode, "div", {"class": "pagenavbar"})
        commonlib.Etree.clean_elements(contentNode, "table", {"class": "pagenav"})
        contentText = html2text.html2text(commonlib.Etree.tostring(contentNode), textonly=True).encode('utf-8').strip()
        print contentText
        if not br:
            title = "{0} ({1})".format(titlefm, c)
            c += 1
        contentText = re.sub(r"\n", "<br />", contentText)
        data.append({'title': title, 'content': contentText})
        if br: break
    
    return data


def getListStories(url):
    tree = commonlib.loadweb(url).build_tree()
    data = []
    for node in tree.xpath("//table[@class='category']/tr/td[2]/a"):
        title = commonlib.stringify(node)
        print '----------------------------------', "\n"
        print title
        print '----------------------------------', "\n"
        link = urljoin(url, node.get('href'))
        data.append({'title': title, 'url': link})
    return data

def appendStories2DB():
    url = 'http://www.truyenviet.com/truyen-nguoi-lon/8-c'
    database = '/longhoanggiang/database/tnl'
    stories = getListStories(url)
    smodel = StoryModel(database)
    smodel.open()
    for story in stories:
        data = getDetail(story['url'], story['title'])
        for chap in data:
            if chap['title'].strip() == '' or chap['content'] == '': continue
            title = commonlib.toUpper(chap['title'])
            content = "<b>{0}</b><br /><hr /><br />{1}".format(title, chap['content'])
            smodel.add_story(title, content, 0)
    smodel.close()
            
def getListTruyen():
    url = 'http://www.truyenviet.com/truyen-nguoi-lon'
    
    tree = commonlib.loadweb(url).build_tree()
    smodel = StoryModel("/longhoanggiang/database/truyennguoilon")
    smodel.open(True)
    for node in tree.xpath("//div[@class='contentdescription']/following-sibling::ul/li/a"):
        cat = commonlib.stringify(node)
        if 'Đồng Tình Luyến' in cat: continue
        print '---------------------------------'
        print cat
        print '---------------------------------'
        link = urljoin(url, node.get('href'))
        
        tree = commonlib.loadweb(link).build_tree()
        for node in tree.xpath("//table[@class='category']/tr/td[2]/a"):
            title = commonlib.stringify(node)
            print '----------------------------------', "\n"
            print title
            print '----------------------------------', "\n"
            link = urljoin(link, node.get('href'))
            data = getDetail(link, title)
            for chap in data:
                if chap['title'].strip() == '' or chap['content'] == '': continue
                title = commonlib.toUpper(chap['title'])
                content = "<b>{0}</b><br /><hr /><br />{1}".format(title, chap['content'])
                smodel.add_story(title, content, 0)
            
    smodel.close()
    

if __name__ == '__main__':
    
#    appendStories2DB()
    getListTruyen()
#    a = getDetail('http://truyenviet.com/truyen-nguoi-lon/5-0-9/760-tnl-1001-dem-x', 'hehe')
#    for i in a:
#        print i
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)